{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:1r8z0dcj) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<br/>Waiting for W&B process to finish, PID 0... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Control-C detected -- Run data was not synced\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Successfully finished last run (ID:1r8z0dcj). Initializing new run:<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import wandb\n",
    "import pandas as pd\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import torch\n",
    "from pandas import DataFrame\n",
    "from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\n",
    "import numpy as np\n",
    "from transformers import EvalPrediction\n",
    "from typing import Optional\n",
    "from transformers import (\n",
    "    Trainer,\n",
    "    BertTokenizer,\n",
    "    TrainingArguments,\n",
    "    HfArgumentParser,\n",
    "    BertForSequenceClassification,\n",
    "    EarlyStoppingCallback\n",
    ")\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "os.environ[\"WANDB_START_METHOD\"] = \"thread\"\n",
    "wandb.init(project=\"hyper_demo\", \n",
    "           tags=[\"baseline\", \"hyperparam\"],\n",
    "           group=\"group_2\")\n",
    "\n",
    "\n",
    "source_data_path = \"/data/tmp/nlp-data/open_source_data/classification/ChnSentiCorp_htl_all.csv\"\n",
    "ber_name_path = \"/data/tmp/christmas.wang/chinese_wwm_ext_pytorch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>CBD中心,周围没什么店铺,说5星有点勉强.不知道为什么卫生间没有电吹风</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7761</th>\n",
       "      <td>0</td>\n",
       "      <td>尼斯酒店的几大特点：噪音大、环境差、配置低、服务效率低。如：1、隔壁歌厅的声音闹至午夜3点许...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7762</th>\n",
       "      <td>0</td>\n",
       "      <td>盐城来了很多次，第一次住盐阜宾馆，我的确很失望整个墙壁黑咕隆咚的，好像被烟熏过一样家具非常的...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7763</th>\n",
       "      <td>0</td>\n",
       "      <td>看照片觉得还挺不错的，又是4星级的，但入住以后除了后悔没有别的，房间挺大但空空的，早餐是有但...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7764</th>\n",
       "      <td>0</td>\n",
       "      <td>我们去盐城的时候那里的最低气温只有4度，晚上冷得要死，居然还不开空调，投诉到酒店客房部，得到...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7765</th>\n",
       "      <td>0</td>\n",
       "      <td>说实在的我很失望，之前看了其他人的点评后觉得还可以才去的，结果让我们大跌眼镜。我想这家酒店以...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7765 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "0         1  距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较...\n",
       "1         1                       商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!\n",
       "2         1         早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。\n",
       "3         1  宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小...\n",
       "4         1               CBD中心,周围没什么店铺,说5星有点勉强.不知道为什么卫生间没有电吹风\n",
       "...     ...                                                ...\n",
       "7761      0  尼斯酒店的几大特点：噪音大、环境差、配置低、服务效率低。如：1、隔壁歌厅的声音闹至午夜3点许...\n",
       "7762      0  盐城来了很多次，第一次住盐阜宾馆，我的确很失望整个墙壁黑咕隆咚的，好像被烟熏过一样家具非常的...\n",
       "7763      0  看照片觉得还挺不错的，又是4星级的，但入住以后除了后悔没有别的，房间挺大但空空的，早餐是有但...\n",
       "7764      0  我们去盐城的时候那里的最低气温只有4度，晚上冷得要死，居然还不开空调，投诉到酒店客房部，得到...\n",
       "7765      0  说实在的我很失望，之前看了其他人的点评后觉得还可以才去的，结果让我们大跌眼镜。我想这家酒店以...\n",
       "\n",
       "[7765 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_df = pd.read_csv(source_data_path, encoding=\"utf-8\")\n",
    "source_df.dropna(how=\"any\", axis=0, inplace=True)\n",
    "source_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6973</th>\n",
       "      <td>0</td>\n",
       "      <td>一、总体来讲，尚可。因为是2007年新开业的酒店，各项硬件设施还能将就。二、浴室及卫生间的设...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1143</th>\n",
       "      <td>1</td>\n",
       "      <td>注意啦！！这个酒店只有6、7楼重新装修过（地毯是深灰色）。其他都是旧装修。我们出住时，提出要...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6174</th>\n",
       "      <td>0</td>\n",
       "      <td>在携程订威龙原本看上了两点，一个是四星级别、一个是香港公司管理。住过之后发现名不副实。首先服...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2883</th>\n",
       "      <td>1</td>\n",
       "      <td>比较高的性价比,环境服务都不错.68的自助餐也还不错,每人给10块钱的抵现券,58/人.就是...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1301</th>\n",
       "      <td>1</td>\n",
       "      <td>不错,就是院子小了一些,到海滩需要走一段端小路,奇怪的是青岛所有酒店都没有自己的海滩</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1200</th>\n",
       "      <td>1</td>\n",
       "      <td>我已经是三年第三次通过携程预订这间酒店了,这次有些失望,酒店的服务态度还是很好,无可挑剔,可...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4464</th>\n",
       "      <td>1</td>\n",
       "      <td>房间蛮合心意，早餐也很丰富，附属设施一般。周围的环境比较差，不过还算有几个小吃店能对付一顿简餐。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店正在申定五星中，不过目前看来四星都有点勉强。大堂很气派，不过细节很粗糙。硬件需要加强。服...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>932</th>\n",
       "      <td>1</td>\n",
       "      <td>服务算可以设施老旧空调太吵在济南，也没别的好选择</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4606</th>\n",
       "      <td>1</td>\n",
       "      <td>非常喜欢居家的感觉，尽管逗留的时间只有两天，很多设备都用不上，但全套餐具、厨具以及餐桌、餐椅...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7765 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "6973      0  一、总体来讲，尚可。因为是2007年新开业的酒店，各项硬件设施还能将就。二、浴室及卫生间的设...\n",
       "1143      1  注意啦！！这个酒店只有6、7楼重新装修过（地毯是深灰色）。其他都是旧装修。我们出住时，提出要...\n",
       "6174      0  在携程订威龙原本看上了两点，一个是四星级别、一个是香港公司管理。住过之后发现名不副实。首先服...\n",
       "2883      1  比较高的性价比,环境服务都不错.68的自助餐也还不错,每人给10块钱的抵现券,58/人.就是...\n",
       "1301      1         不错,就是院子小了一些,到海滩需要走一段端小路,奇怪的是青岛所有酒店都没有自己的海滩\n",
       "...     ...                                                ...\n",
       "1200      1  我已经是三年第三次通过携程预订这间酒店了,这次有些失望,酒店的服务态度还是很好,无可挑剔,可...\n",
       "4464      1   房间蛮合心意，早餐也很丰富，附属设施一般。周围的环境比较差，不过还算有几个小吃店能对付一顿简餐。\n",
       "17        1  酒店正在申定五星中，不过目前看来四星都有点勉强。大堂很气派，不过细节很粗糙。硬件需要加强。服...\n",
       "932       1                           服务算可以设施老旧空调太吵在济南，也没别的好选择\n",
       "4606      1  非常喜欢居家的感觉，尽管逗留的时间只有两天，很多设备都用不上，但全套餐具、厨具以及餐桌、餐椅...\n",
       "\n",
       "[7765 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_df = shuffle(source_df)\n",
    "source_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6973</th>\n",
       "      <td>0</td>\n",
       "      <td>一、总体来讲，尚可。因为是2007年新开业的酒店，各项硬件设施还能将就。二、浴室及卫生间的设...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1143</th>\n",
       "      <td>1</td>\n",
       "      <td>注意啦！！这个酒店只有6、7楼重新装修过（地毯是深灰色）。其他都是旧装修。我们出住时，提出要...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6174</th>\n",
       "      <td>0</td>\n",
       "      <td>在携程订威龙原本看上了两点，一个是四星级别、一个是香港公司管理。住过之后发现名不副实。首先服...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2883</th>\n",
       "      <td>1</td>\n",
       "      <td>比较高的性价比,环境服务都不错.68的自助餐也还不错,每人给10块钱的抵现券,58/人.就是...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1301</th>\n",
       "      <td>1</td>\n",
       "      <td>不错,就是院子小了一些,到海滩需要走一段端小路,奇怪的是青岛所有酒店都没有自己的海滩</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>1</td>\n",
       "      <td>五月底入住的该宾馆，对于三星来说还是可以的，价钱不贵，可能它的位置不是繁华地吧。就是前台的收...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7651</th>\n",
       "      <td>0</td>\n",
       "      <td>其他没什么，就是房价便宜。很老的星级宾馆了，愧对4星的标准了。补充点评2008年5月31日：...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5001</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店房间很好，浴室在房间中间，但要小心一定要把窗帘拉下，对面就是办公楼。酒店服务非常到位。绝...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2400</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店设施虽然有很多创新，但是并不为使用者着想的。窗帘自动开关虽然好，但是却不太好用。还有控制...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4396</th>\n",
       "      <td>1</td>\n",
       "      <td>总体感觉不错，作为休闲度假还是很好的选择，度假山庄的环境很好，但是携程提供的选择太少了，只有...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1600 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "6973      0  一、总体来讲，尚可。因为是2007年新开业的酒店，各项硬件设施还能将就。二、浴室及卫生间的设...\n",
       "1143      1  注意啦！！这个酒店只有6、7楼重新装修过（地毯是深灰色）。其他都是旧装修。我们出住时，提出要...\n",
       "6174      0  在携程订威龙原本看上了两点，一个是四星级别、一个是香港公司管理。住过之后发现名不副实。首先服...\n",
       "2883      1  比较高的性价比,环境服务都不错.68的自助餐也还不错,每人给10块钱的抵现券,58/人.就是...\n",
       "1301      1         不错,就是院子小了一些,到海滩需要走一段端小路,奇怪的是青岛所有酒店都没有自己的海滩\n",
       "...     ...                                                ...\n",
       "763       1  五月底入住的该宾馆，对于三星来说还是可以的，价钱不贵，可能它的位置不是繁华地吧。就是前台的收...\n",
       "7651      0  其他没什么，就是房价便宜。很老的星级宾馆了，愧对4星的标准了。补充点评2008年5月31日：...\n",
       "5001      1  酒店房间很好，浴室在房间中间，但要小心一定要把窗帘拉下，对面就是办公楼。酒店服务非常到位。绝...\n",
       "2400      1  酒店设施虽然有很多创新，但是并不为使用者着想的。窗帘自动开关虽然好，但是却不太好用。还有控制...\n",
       "4396      1  总体感觉不错，作为休闲度假还是很好的选择，度假山庄的环境很好，但是携程提供的选择太少了，只有...\n",
       "\n",
       "[1600 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df = source_df[0:1600]\n",
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1618</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店离火车站不远,打的只要8元。就在金山寺边上，人工湖还和风景区相通，可惜没找到入口，否则省...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2278</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店门脸有点小，你们倒是很大，房间还不错！就是房间的门已经有些下沉了，关门一定要用点力量，早...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4045</th>\n",
       "      <td>1</td>\n",
       "      <td>还可以，离汽车站也很近。离轮渡也很近，也比较干净。起先我预定了厦门元利酒店，结果上网还有时间...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6623</th>\n",
       "      <td>0</td>\n",
       "      <td>手机已收到短消息回复订房OK，前台自己没查到，却叫我打电话给携程，要我自己搞定，即使我给前台...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6968</th>\n",
       "      <td>0</td>\n",
       "      <td>地理位置比较方便,但酒店服务态度不好,环境普通</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1200</th>\n",
       "      <td>1</td>\n",
       "      <td>我已经是三年第三次通过携程预订这间酒店了,这次有些失望,酒店的服务态度还是很好,无可挑剔,可...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4464</th>\n",
       "      <td>1</td>\n",
       "      <td>房间蛮合心意，早餐也很丰富，附属设施一般。周围的环境比较差，不过还算有几个小吃店能对付一顿简餐。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店正在申定五星中，不过目前看来四星都有点勉强。大堂很气派，不过细节很粗糙。硬件需要加强。服...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>932</th>\n",
       "      <td>1</td>\n",
       "      <td>服务算可以设施老旧空调太吵在济南，也没别的好选择</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6164 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "1618      1  酒店离火车站不远,打的只要8元。就在金山寺边上，人工湖还和风景区相通，可惜没找到入口，否则省...\n",
       "1         1                       商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!\n",
       "2278      1  酒店门脸有点小，你们倒是很大，房间还不错！就是房间的门已经有些下沉了，关门一定要用点力量，早...\n",
       "4045      1  还可以，离汽车站也很近。离轮渡也很近，也比较干净。起先我预定了厦门元利酒店，结果上网还有时间...\n",
       "6623      0  手机已收到短消息回复订房OK，前台自己没查到，却叫我打电话给携程，要我自己搞定，即使我给前台...\n",
       "...     ...                                                ...\n",
       "6968      0                            地理位置比较方便,但酒店服务态度不好,环境普通\n",
       "1200      1  我已经是三年第三次通过携程预订这间酒店了,这次有些失望,酒店的服务态度还是很好,无可挑剔,可...\n",
       "4464      1   房间蛮合心意，早餐也很丰富，附属设施一般。周围的环境比较差，不过还算有几个小吃店能对付一顿简餐。\n",
       "17        1  酒店正在申定五星中，不过目前看来四星都有点勉强。大堂很气派，不过细节很粗糙。硬件需要加强。服...\n",
       "932       1                           服务算可以设施老旧空调太吵在济南，也没别的好选择\n",
       "\n",
       "[6164 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = source_df[1600:-1]\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le = LabelEncoder()\n",
    "le.fit(source_df[\"label\"].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels=None):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        if self.labels:\n",
    "            item[\"labels\"] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.encodings[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_2_dataset(df: DataFrame, le: LabelEncoder, text: str, label: str, tokenizer) -> Dataset:\n",
    "    \"\"\"Transform DataFrame into DataSet\n",
    "\n",
    "    Args:\n",
    "        df (DataFrame): Source Data\n",
    "        le (LabelEncoder): LabelEncoder\n",
    "        text (str): Column Name of Text\n",
    "        label (str): Column Name of Label\n",
    "        tokenizer ([type]): Tokenizer\n",
    "\n",
    "    Returns:\n",
    "        Dataset: DataSet\n",
    "    \"\"\"\n",
    "    x = list(df[text])\n",
    "    df[\"label_id\"] = le.transform(df[label].tolist())\n",
    "    y = list(df[\"label_id\"])\n",
    "    x_tokenized = tokenizer(x, padding=True, truncation=True, max_length=512)\n",
    "    result = Dataset(x_tokenized, y)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(ber_name_path, do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_25336/3498692374.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"label_id\"] = le.transform(df[label].tolist())\n"
     ]
    }
   ],
   "source": [
    "train_dataset = df_2_dataset(train_df, le, \"review\", \"label\", tokenizer)\n",
    "eval_dataset = df_2_dataset(eval_df, le, \"review\", \"label\", tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(results: EvalPrediction) -> Optional[dict]:\n",
    "    \"\"\"Calculate Metrics\n",
    "\n",
    "    Args:\n",
    "        results (EvalPrediction): Predictions & labels\n",
    "\n",
    "    Returns:\n",
    "        [type]: Dictionary of Metrics\n",
    "    \"\"\"\n",
    "    pred, labels = results.predictions, results.label_ids\n",
    "    pred = np.argmax(pred, axis=1)\n",
    "\n",
    "    accuracy = accuracy_score(y_true=labels, y_pred=pred)\n",
    "    recall = recall_score(y_true=labels, y_pred=pred, average=\"binary\")\n",
    "    precision = precision_score(y_true=labels, y_pred=pred, average=\"binary\")\n",
    "    f1 = f1_score(y_true=labels, y_pred=pred, average=\"binary\")\n",
    "    \n",
    "    return {\"accuracy\": accuracy, \"precision\": precision, \"recall\": recall, \"f1\": f1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_dataset: Dataset,\n",
    "          valid_dataset: Dataset,\n",
    "          pre_model_path: str\n",
    "          ) -> None:\n",
    "    args = TrainingArguments(\n",
    "        report_to=\"wandb\",\n",
    "        output_dir=\"/data/christmas.wang/project/classification_base_project/output\",\n",
    "        evaluation_strategy=\"steps\",\n",
    "        eval_steps=500,\n",
    "        per_device_train_batch_size=16,\n",
    "        per_device_eval_batch_size=16,\n",
    "        learning_rate=2e-5,\n",
    "        num_train_epochs=6,\n",
    "        seed=0,\n",
    "        load_best_model_at_end=True,\n",
    "    )\n",
    "    \n",
    "    def model_init():\n",
    "        return BertForSequenceClassification.from_pretrained(pre_model_path, num_labels=2)\n",
    "               \n",
    "    trainer = Trainer(\n",
    "        model_init=model_init,\n",
    "        args=args,\n",
    "        train_dataset=train_dataset,\n",
    "        eval_dataset=valid_dataset,\n",
    "        compute_metrics=compute_metrics,\n",
    "        callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],\n",
    "    )\n",
    "    def hp_space(trial):\n",
    "        return {\n",
    "            \"learning_rate\": trial.suggest_float(\"learning_rate\",1e-5, 5e-5, log=True),\n",
    "            \"weight_decay\": trial.suggest_float(\"weight_decay\", 0.01, 0.3),\n",
    "            \"num_train_epochs\": trial.suggest_int(\"num_train_epochs\", 2, 8),\n",
    "            \"seed\": trial.suggest_int(\"seed\", 20, 40),\n",
    "            \"per_device_train_batch_size\": trial.suggest_categorical(\"per_device_train_batch_size\", [4, 8, 16]),\n",
    "        }\n",
    "    best_trial = trainer.hyperparameter_search(\n",
    "        direction=\"maximize\",\n",
    "        backend=\"optuna\",\n",
    "        n_trials=10,\n",
    "        hp_space=hp_space\n",
    "    )\n",
    "    print(\"*************************************\")\n",
    "    print(\" Best run %s\" % str(best_trial))\n",
    "    print(\"*************************************\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "loading configuration file /data/tmp/christmas.wang/chinese_wwm_ext_pytorch/config.json\n",
      "Model config BertConfig {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.10.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "loading weights file /data/tmp/christmas.wang/chinese_wwm_ext_pytorch/pytorch_model.bin\n",
      "Some weights of the model checkpoint at /data/tmp/christmas.wang/chinese_wwm_ext_pytorch were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at /data/tmp/christmas.wang/chinese_wwm_ext_pytorch and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "\u001b[32m[I 2021-10-18 11:52:24,515]\u001b[0m A new study created in memory with name: no-name-ca93a6a2-7aef-4430-9b0f-4f46f387620b\u001b[0m\n",
      "Trial:\n",
      "loading configuration file /data/tmp/christmas.wang/chinese_wwm_ext_pytorch/config.json\n",
      "Model config BertConfig {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.10.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "loading weights file /data/tmp/christmas.wang/chinese_wwm_ext_pytorch/pytorch_model.bin\n",
      "Some weights of the model checkpoint at /data/tmp/christmas.wang/chinese_wwm_ext_pytorch were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at /data/tmp/christmas.wang/chinese_wwm_ext_pytorch and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "***** Running training *****\n",
      "  Num examples = 6164\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 291\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<br/>Waiting for W&B process to finish, PID 0... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Network error resolved after 0:01:35.727296, resuming normal operation.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Network error resolved after 0:03:37.356086, resuming normal operation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "387d1482012c4b84a5066bbe26307076",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\">\n",
       "</div><div class=\"wandb-col\">\n",
       "</div></div>\n",
       "Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
       "<br/>Synced <strong style=\"color:#cdcd00\">./output</strong>: <a href=\"https://wandb.ai/8christmas8/hyper_demo/runs/11re4taw\" target=\"_blank\">https://wandb.ai/8christmas8/hyper_demo/runs/11re4taw</a><br/>\n",
       "Find logs at: <code>./wandb/run-20211018_114835-11re4taw/logs</code><br/>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                    Syncing run <strong><a href=\"https://wandb.ai/8christmas8/hyper_demo/runs/1r8z0dcj\" target=\"_blank\">./output</a></strong> to <a href=\"https://wandb.ai/8christmas8/hyper_demo\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
       "\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[33m[W 2021-10-18 11:56:45,713]\u001b[0m Trial 0 failed because of the following error: RuntimeError('Caught RuntimeError in replica 1 on device 1.\\nOriginal Traceback (most recent call last):\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py\", line 61, in _worker\\n    output = module(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 1524, in forward\\n    outputs = self.bert(\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 990, in forward\\n    encoder_outputs = self.encoder(\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 582, in forward\\n    layer_outputs = layer_module(\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 470, in forward\\n    self_attention_outputs = self.attention(\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 401, in forward\\n    self_outputs = self.self(\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 333, in forward\\n    attention_probs = self.dropout(attention_probs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\\n    return forward_call(*input, **kwargs)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/dropout.py\", line 58, in forward\\n    return F.dropout(input, self.p, self.training, self.inplace)\\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/functional.py\", line 1168, in dropout\\n    return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)\\nRuntimeError: CUDA out of memory. Tried to allocate 96.00 MiB (GPU 1; 31.75 GiB total capacity; 1.54 GiB already allocated; 40.00 MiB free; 1.66 GiB reserved in total by PyTorch)\\n')\u001b[0m\n",
      "Traceback (most recent call last):\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/optuna/study/_optimize.py\", line 213, in _run_trial\n",
      "    value_or_values = func(trial)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/integrations.py\", line 140, in _objective\n",
      "    trainer.train(resume_from_checkpoint=checkpoint, trial=trial)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\", line 1284, in train\n",
      "    tr_loss += self.training_step(model, inputs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\", line 1789, in training_step\n",
      "    loss = self.compute_loss(model, inputs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\", line 1821, in compute_loss\n",
      "    outputs = model(**inputs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py\", line 168, in forward\n",
      "    outputs = self.parallel_apply(replicas, inputs, kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py\", line 178, in parallel_apply\n",
      "    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py\", line 86, in parallel_apply\n",
      "    output.reraise()\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/_utils.py\", line 425, in reraise\n",
      "    raise self.exc_type(msg)\n",
      "RuntimeError: Caught RuntimeError in replica 1 on device 1.\n",
      "Original Traceback (most recent call last):\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py\", line 61, in _worker\n",
      "    output = module(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 1524, in forward\n",
      "    outputs = self.bert(\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 990, in forward\n",
      "    encoder_outputs = self.encoder(\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 582, in forward\n",
      "    layer_outputs = layer_module(\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 470, in forward\n",
      "    self_attention_outputs = self.attention(\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 401, in forward\n",
      "    self_outputs = self.self(\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 333, in forward\n",
      "    attention_probs = self.dropout(attention_probs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/dropout.py\", line 58, in forward\n",
      "    return F.dropout(input, self.p, self.training, self.inplace)\n",
      "  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/functional.py\", line 1168, in dropout\n",
      "    return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)\n",
      "RuntimeError: CUDA out of memory. Tried to allocate 96.00 MiB (GPU 1; 31.75 GiB total capacity; 1.54 GiB already allocated; 40.00 MiB free; 1.66 GiB reserved in total by PyTorch)\n",
      "\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Caught RuntimeError in replica 1 on device 1.\nOriginal Traceback (most recent call last):\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py\", line 61, in _worker\n    output = module(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 1524, in forward\n    outputs = self.bert(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 990, in forward\n    encoder_outputs = self.encoder(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 582, in forward\n    layer_outputs = layer_module(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 470, in forward\n    self_attention_outputs = self.attention(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 401, in forward\n    self_outputs = self.self(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 333, in forward\n    attention_probs = self.dropout(attention_probs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/dropout.py\", line 58, in forward\n    return F.dropout(input, self.p, self.training, self.inplace)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/functional.py\", line 1168, in dropout\n    return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)\nRuntimeError: CUDA out of memory. Tried to allocate 96.00 MiB (GPU 1; 31.75 GiB total capacity; 1.54 GiB already allocated; 40.00 MiB free; 1.66 GiB reserved in total by PyTorch)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_25336/2918764716.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m train(train_dataset,\n\u001b[0m\u001b[1;32m      2\u001b[0m       \u001b[0meval_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m       ber_name_path)\n",
      "\u001b[0;32m/tmp/ipykernel_25336/2257253479.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(train_dataset, valid_dataset, pre_model_path)\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0;34m\"per_device_train_batch_size\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuggest_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"per_device_train_batch_size\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m16\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         }\n\u001b[0;32m---> 37\u001b[0;31m     best_trial = trainer.hyperparameter_search(\n\u001b[0m\u001b[1;32m     38\u001b[0m         \u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"maximize\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mbackend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"optuna\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mhyperparameter_search\u001b[0;34m(self, hp_space, compute_objective, n_trials, direction, backend, hp_name, **kwargs)\u001b[0m\n\u001b[1;32m   1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m         \u001b[0mrun_hp_search\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_hp_search_optuna\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mHPSearchBackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOPTUNA\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mrun_hp_search_ray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m         \u001b[0mbest_run\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_hp_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdirection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1710\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhp_search_backend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/integrations.py\u001b[0m in \u001b[0;36mrun_hp_search_optuna\u001b[0;34m(trainer, n_trials, direction, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"n_jobs\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0mstudy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_study\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdirection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m     \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_objective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0mbest_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mBestRun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_trial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_trial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_trial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    398\u001b[0m             )\n\u001b[1;32m    399\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m         _optimize(\n\u001b[0m\u001b[1;32m    401\u001b[0m             \u001b[0mstudy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             _optimize_sequential(\n\u001b[0m\u001b[1;32m     67\u001b[0m                 \u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m                 \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mTrialState\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFAIL\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfunc_err\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc_err\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mfunc_err\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    265\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m         \u001b[0mvalue_or_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    214\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0;31m# TODO(mamu): Handle multi-objective cases.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/integrations.py\u001b[0m in \u001b[0;36m_objective\u001b[0;34m(trial, checkpoint_dir)\u001b[0m\n\u001b[1;32m    138\u001b[0m                     \u001b[0mcheckpoint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m         \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobjective\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m         \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m         \u001b[0;31m# If there hasn't been any evaluation during the training loop.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"objective\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1282\u001b[0m                         \u001b[0mtr_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1283\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1284\u001b[0;31m                     \u001b[0mtr_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1285\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_flos\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloating_point_ops\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   1787\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1788\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1789\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1790\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1791\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_gpu\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mcompute_loss\u001b[0;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[1;32m   1819\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1820\u001b[0m             \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1821\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1822\u001b[0m         \u001b[0;31m# Save past state if it exists\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1823\u001b[0m         \u001b[0;31m# TODO: this needs to be fixed and made cleaner later.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m             \u001b[0mreplicas\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_ids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparallel_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    169\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py\u001b[0m in \u001b[0;36mparallel_apply\u001b[0;34m(self, replicas, inputs, kwargs)\u001b[0m\n\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mparallel_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mparallel_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_ids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplicas\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py\u001b[0m in \u001b[0;36mparallel_apply\u001b[0;34m(modules, inputs, kwargs_tup, devices)\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExceptionWrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m             \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m         \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/_utils.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    423\u001b[0m             \u001b[0;31m# have message field\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    424\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Caught RuntimeError in replica 1 on device 1.\nOriginal Traceback (most recent call last):\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py\", line 61, in _worker\n    output = module(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 1524, in forward\n    outputs = self.bert(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 990, in forward\n    encoder_outputs = self.encoder(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 582, in forward\n    layer_outputs = layer_module(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 470, in forward\n    self_attention_outputs = self.attention(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 401, in forward\n    self_outputs = self.self(\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py\", line 333, in forward\n    attention_probs = self.dropout(attention_probs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1051, in _call_impl\n    return forward_call(*input, **kwargs)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/modules/dropout.py\", line 58, in forward\n    return F.dropout(input, self.p, self.training, self.inplace)\n  File \"/data/Miniconda3/envs/wf_pytorch_env/lib/python3.8/site-packages/torch/nn/functional.py\", line 1168, in dropout\n    return _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)\nRuntimeError: CUDA out of memory. Tried to allocate 96.00 MiB (GPU 1; 31.75 GiB total capacity; 1.54 GiB already allocated; 40.00 MiB free; 1.66 GiB reserved in total by PyTorch)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Network error (ConnectTimeout), entering retry loop.\n"
     ]
    }
   ],
   "source": [
    "    train(train_dataset,\n",
    "          eval_dataset,\n",
    "          ber_name_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
